Apache Airflow QuickStartを実行してみた

AWSTemplateFormatVersion: 2010-09-09

Parameters:
  VpcId:
    Type: String
    Default: vpc-xxxx
  SubnetId:
    Type: String
    Default: subnet-xxxx
  AccessCidr:
    Type: String
    Default: x.x.x.x/32
  KeyName:
    Type: AWS::EC2::KeyPair::KeyName
    Default: test-key
  AirflowPort:
    Type: String
    Default: 8080
  AmazonLinuxImageId:
    Type: AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>
    Default: /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2
    Description: The AmazonLinux2 image id at least.
  InstanceType:
    Type: String
    Default: t3.micro
    AllowedValues:
      - t3.nano
      - t3.micro
      - t3.small    
  TimeZone:
    Type: String
    Default: Asia/Tokyo
  Locale:
    Type: String
    Default: ja_JP.utf8
  AirflowVersion:
    Type: String
    Default: 2.2.3

Metadata: 
  AWS::CloudFormation::Interface: 
    ParameterGroups: 
      - Label: 
          default: Airflow init parameters
        Parameters: 
          - VpcId
          - SubnetId
          - AccessCidr
          - KeyName

Resources:
  SecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties: 
      VpcId: !Ref VpcId
      GroupName: airflow-localaccess-sg
      GroupDescription: !Sub localaccess-sg-${AWS::StackName}
      Tags:
        - Key: Name
          Value: airflow-localaccess-sg
      SecurityGroupIngress:
        - IpProtocol: tcp
          FromPort: !Ref AirflowPort
          ToPort: !Ref AirflowPort
          CidrIp: !Ref AccessCidr
        - IpProtocol: tcp
          FromPort: 22
          ToPort: 22
          CidrIp: !Ref AccessCidr

  EC2Role:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Action: 'sts:AssumeRole'
          Effect: Allow
          Principal:
            Service: ec2.amazonaws.com
      MaxSessionDuration: 36000
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore
      Tags:
        - Key: Name
          Value: role-ec2-airflow

  IAMInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Roles: 
        - !Ref EC2Role
      Path: "/"

  AirflowServer:
    Type: AWS::EC2::Instance
    CreationPolicy: 
      ResourceSignal:
        Timeout: PT20M
    Properties: 
      IamInstanceProfile: !Ref IAMInstanceProfile
      ImageId: !Ref AmazonLinuxImageId
      InstanceType: !Ref InstanceType
      KeyName: !Ref KeyName
      InstanceInitiatedShutdownBehavior: stop
      DisableApiTermination: false
      Monitoring: False
      BlockDeviceMappings:    
        - DeviceName: /dev/xvda
          Ebs:
            VolumeType: gp3
            DeleteOnTermination: true
            Encrypted: true
            VolumeSize: 10
      NetworkInterfaces:
        - AssociatePublicIpAddress: true
          DeviceIndex: 0
          SubnetId: !Ref SubnetId
          GroupSet: 
            - !Ref SecurityGroup
      Tags: 
        - Key: Name
          Value: airflow-server
      UserData: 
        Fn::Base64: !Sub |
          #!/bin/bash
          echo "--------- Change Timezone/Locale ------------------------"
          timedatectl set-timezone ${TimeZone}
          localectl set-locale LANG=${Locale}

          echo "--------- Install & Configure Python --------------------"
          amazon-linux-extras enable python3.8
          yum -y install python3.8
          update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1
          /usr/bin/pip3.8 install --upgrade pip
          update-alternatives --install /usr/bin/pip pip /usr/local/bin/pip3 1  
          python -V
          pip -V

          echo "--------- Change yum configuration ----------------------"
          sed -i -e '1s/python/python2/g' /bin/yum
          sed -i -e '1s/python/python2/g' /usr/libexec/urlgrabber-ext-down
          yum update -y

          echo "--------- Update Sqlite for airflow requirement ---------"
          yum install -y gcc-c++ python-devel python-setuptools

          wget https://www.sqlite.org/2021/sqlite-autoconf-3370000.tar.gz
          tar xzvf sqlite-autoconf-3370000.tar.gz
          cd sqlite-autoconf-3370000
          ./configure --prefix=/opt/sqlite/sqlite3
          make
          make install

          echo "--------- Install Airflow -------------------------------"
          groupadd airflow
          useradd airflow -g airflow
          passwd -d airflow
          su airflow
          cd ~
          echo 'export PATH="$PATH:/home/airflow/.local/bin"' >> ~/.bashrc
          echo 'export AIRFLOW_HOME=~/airflow' >> ~/.bashrc
          echo 'export LD_LIBRARY_PATH="/opt/sqlite/sqlite3/lib"' >> ~/.bashrc
          echo 'export LD_RUN_PATH="/opt/sqlite/sqlite3/lib"' >> ~/.bashrc
          source ~/.bashrc

          sqlite3 --version
          python -c "import sqlite3; print(sqlite3.sqlite_version)"
          
          mkdir ~/airflow
          AIRFLOW_VERSION=${AirflowVersion}
          PYTHON_VERSION="$(python --version | cut -d " " -f 2 | cut -d "." -f 1-2)"
          CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-$AIRFLOW_VERSION/constraints-$PYTHON_VERSION.txt"
          pip install "apache-airflow==$AIRFLOW_VERSION" --constraint "$CONSTRAINT_URL"

          echo "<EXECUTE> /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource AirflowServer --region ${AWS::Region}"
          /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource AirflowServer --region ${AWS::Region}

Outputs:
  AccessPublicIp:
    Value: !GetAtt AirflowServer.PublicIp
  AirflowURL:
    Value: !Sub "http://${AirflowServer.PublicIp}:${AirflowPort}"